from typing import Iterator

from langchain_core.document_loaders import BaseBlobParser
from langchain_core.documents import Document
from langchain_core.documents.base import Blob


class CustomParaser(BaseBlobParser):
    """自定义Blob 文档加载器"""
    def lazy_parse(self, blob: Blob) -> Iterator[Document]:
        line_number = 0
        with blob.as_bytes_io() as f:
            for one_line in f:
                yield Document(
                    page_content= one_line,
                    metadata = {"source":blob.source,"line_number":line_number})
                line_number += 1

blob = Blob.from_path("./eshop_goods.txt") #读取数据
# blob = Blob(data="hello \r\n world") #读取数据  Blob需要调试

parser = CustomParaser()
docs = list(parser.lazy_parse(blob)) #解析数据

for onedoc in docs:
    print(onedoc)

print(len(docs))